# -*- coding: utf-8 -*-
""" 爬取房源数据 """

import requests
from lxml import etree
import pandas as pd
import time

# 1. 设置行不限制数量
pd.set_option('display.max_rows',None)
# 2. 设置列不限制数量
pd.set_option('display.max_columns',None)
# url = 'https://gz.lianjia.com/ershoufang/pg{页码}rs{地区}/'
data_lis = []

def spider_house(pg,rs="佛山"):
    # pg = 2
    # rs = "佛山"
    url = f'https://gz.lianjia.com/ershoufang/pg{pg}rs{rs}/'
    response = requests.get(url=url,headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64)  like Gecko'})
    html = etree.HTML(response.text)
    title = html.xpath("//div[@class='title']/a/text()")
    # print(title)
    # print(len(title))

    houseInfo = html.xpath('//div[@class="houseInfo"]//text()')
    # print(houseInfo)
    # print(len(houseInfo))

    totalPrice = html.xpath('//div[@class="totalPrice"]/span/text()')
    # print(totalPrice)
    # print(len(totalPrice))

    unitPrice = html.xpath('//div[@class="unitPrice"]/span/text()')
    # print(unitPrice)
    # print(len(unitPrice))

    #todo：提取houseInfo的各类信息
    house_type,size,direction,renovation,position,ta = [],[],[],[],[],[]
    try:
        for i in houseInfo:
            # i:'1室1厅 | 45.9平米 | 南 北 | 简装 | 低楼层(共8层) | 2006年建 | 塔楼'
            info = i.split("| ")
            house_type.append(info[0])
            size.append(info[1])
            direction.append(info[2])
            renovation.append(info[3])
            position.append(info[4])
            ta.append(info[-1])
    except Exception as e:
        print("数据切片异常：",e)
    # print(house_type)
    # print(ta)

    for title,totalPrice,unitPrice,house_type,size,direction,renovation,position,ta in zip(title,totalPrice,unitPrice,house_type,size,direction,renovation,position,ta):
        data_dict = {}
        data_dict["title"] = title
        data_dict["totalPrice"] = totalPrice
        data_dict["unitPrice"] = unitPrice
        data_dict["house_type"] = house_type
        data_dict["size"] = size
        data_dict["direction"] = direction
        data_dict["renovation"] = renovation
        data_dict["position"] = position
        data_dict["ta"] = ta

        print(data_dict)
        data_lis.append(data_dict)

for i in range(1,34,3):
    spider_house(i)
    time.sleep(5)



df = pd.DataFrame(data_lis,columns=["title","totalPrice","unitPrice","house_type","size","direction","renovation","position","ta"])
# print(df)
df.to_csv('2021年3月30日佛山二手房.csv',index=False)
